package edu.columbia.gate;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Date;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.*;

import gate.*;
import gate.creole.SerialAnalyserController;
import gate.creole.SerialController;
import gate.security.AccessController;
import gate.security.Group;
import gate.security.Session;
import gate.security.User;
import gate.util.persistence.PersistenceManager;

public class GateWrapper
{

	private static final String PROPERTIES_FILE_NAME = "GateWrapper.properties";
	private static final String DATASTORE_URL_PROPERTY_NAME = "datastore.url";
	
	private static final String GATE_USER_PROPERTY_NAME = "gate.user";
	private static final String GATE_GROUP_PROPERTY_NAME = "gate.group";
	private static final String GATE_PASSWORD_PROPERTY_NAME = "gate.password";
	private static final String GATE_APPLICATION_FILE_NAME_PROPERTY_NAME = "application.file.name";
	private static final String GATE_CONFIGURATION_FILE_NAME_PROPERTY_NAME = "gate.config.file.name";
	private static final String LOG_FILE_NAME_PROPERTY_NAME = "log.file";
	private static final String PROCESSING_RESOURCES_PATH_PROPERTY_NAME = "processing.resources.path";
	private static final String USER_PLUGINS_PATH_PROPERTY_NAME = "user.plugins.path";
	private static final int MAX_EXECUTION_TIME_PER_CHARACTER_IN_MILLISECONDS = 300;
	private static final int MAX_DB_ACCESS_OVERHEAD_PER_DOC_IN_MILLIS = 60000;
	
	private static final String datastoreType = "gate.persist.PostgresDataStore";
	
	private static String[] corpusNames;
	private static long[] corpusIds;
	private static String applicationFileName;
	private static String processingResourcesPath;
	private static String userPluginsPath;
	private static String datastoreUrl;
	private static String gateHome;
	private static String gateUser;
	private static String gateGroup;
	private static String gatePassword;
	private static String gateConfigFileName;
	private static String logFile;
	private static ArrayList<Document> documentsSkipped = new ArrayList<Document>();
	
	private static DataStore dataStore;
	
	public static Logger logger;
	
	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception
	{
		try
		{
			readPropertiesFile();
			init();
			logger.info("Application starting.");
			
			parseCommandLineArgs(args);
			initGate();
			
			File applicationFile = new File(processingResourcesPath, applicationFileName);
			if(! applicationFile.exists())
			{
				logger.fatal("Application file " + processingResourcesPath + "/" + applicationFileName + " was not found. Check that both properties, " + PROCESSING_RESOURCES_PATH_PROPERTY_NAME + " and " + GATE_APPLICATION_FILE_NAME_PROPERTY_NAME + " are correctly set.");
				System.exit(1);
			}
			else
			{
				setPathsInFile(applicationFile);
			}
			logger.info("GATE is initialized. Loading application from file " + applicationFile.getAbsolutePath());
			CorpusController application= null;
			try
			{
				application =
			      (CorpusController)PersistenceManager.loadObjectFromFile(applicationFile);
			} catch (Exception ex)
			{
				logger.fatal("Could not load application file " + applicationFile.getAbsolutePath() + ". File exists, but caused an error.", ex);
				System.exit(1);
			}
			
			getDatastoreSession();
			
			logger.info("Initialization succeeded.");
			logger.info("Starting batch processing of corpora.");
			long appStartMillis = System.currentTimeMillis();
			long totalLengthOfCorpus = 0;
			int documentsProcessed = 0;
			for(int i=0; i<corpusNames.length; i++)
			{
				logger.info("Starting processing of corpus " + corpusNames[i] + ", id=" + corpusIds[i] + ".");
				long startMillis = System.currentTimeMillis();
				try
				{
					// Load the corpus into memory.
					Corpus corpus = loadCorpus(corpusIds[i]);
					
					logger.info("Corpus contains " + corpus.size() + " documents.");

					boolean isFirstDoc = true;

					Iterator docIterator = corpus.iterator();
					while(docIterator.hasNext())
					{
						long docStartMillis = System.currentTimeMillis();
						Document doc = (Document) docIterator.next(); 
						int annotCount = doc.getAnnotations().size();
						int featuCount = doc.getFeatures().size();
						long docSize = doc.getContent().size();
						totalLengthOfCorpus += docSize; 
						long maxRuntime = docSize * MAX_EXECUTION_TIME_PER_CHARACTER_IN_MILLISECONDS + MAX_DB_ACCESS_OVERHEAD_PER_DOC_IN_MILLIS;
						if(isFirstDoc)
						{
							isFirstDoc = false;
							// Give 30 extra seconds to allow the application 
							// to load the corpus from the DB.
							maxRuntime += 30000;
						}
						PipelineMonitor monitorThread = null;
						
						logger.info("Starting processing of document '" + doc.getName() + "', lr_id=" + doc.getLRPersistenceId().toString() );
						try
						{
							SerialController singleDocApplication = (SerialController) Factory.createResource("gate.creole.SerialController"
									, Factory.newFeatureMap(), Factory.newFeatureMap(), "SingleDocApp");
							
							monitorThread = new PipelineMonitor(maxRuntime, singleDocApplication);
							
							Iterator iterator = application.getPRs().iterator();
							while(iterator.hasNext())
							{
								ProcessingResource pr = (ProcessingResource) iterator.next();
								pr.setParameterValue("document", doc);
								singleDocApplication.add(pr);
							}
							monitorThread.start();
							singleDocApplication.execute();
							documentsProcessed++;
							if(monitorThread.wasInterruped || singleDocApplication.isInterrupted())
							{
								if(doc != null)
									logger.error("DOCUMENT " + doc.getName() + " WAS SKIPPED DUE TO APPLICATION TIMEOUT.");
								documentsSkipped.add(doc);
							}
							else
							{
								removeAnnotationsFromDoc(doc, "Token SpaceToken TextLine FreeText ByClause Lookup Address ShortDateTime");
								try
								{
									dataStore.sync(doc);
									long docEndMillis = System.currentTimeMillis();
									logger.info("Document '" + doc.getName() + "' processed successfully. " + docSize + " chars in "
											+ (docEndMillis - docStartMillis) + " ms. Avg time per char: " 
											+ (((double)  (docEndMillis - docStartMillis)) / ((double)  docSize) ) 
											+ ". " + documentsProcessed + " documents processed so far. "
											+ " At an average of " + ((docEndMillis - appStartMillis) / documentsProcessed) + " ms per document." 
											//+ "\n\tAnnotations before: " + annotCount + "  Annotations after: " + doc.getAnnotations().size() + "\n"
											//+ "\tFeatures before: " + featuCount + "  Features after: " + doc.getFeatures().size() + ""
											);
								} catch (Exception e)
								{
									if(doc != null) documentsSkipped.add(doc);
									logger.error("Error while saving document '" + doc.getName() + "'.", e);
								}
							}
						}
						catch (Exception docLevelExc)
						{
							if(doc != null) documentsSkipped.add(doc);
							logger.error("Error while processing document '" + doc.getName() + "'.", docLevelExc);
						}
						finally
						{
							try
							{
								doc.cleanup();
								if(monitorThread != null)
									monitorThread.interrupt();
							} catch (Exception e){}
						}
						
					}
					if(application.isInterrupted())
					{
						throw new Exception("Gate application (pipeline) was interrupted while processing corpus " + corpus.getName());
					}

					// Unload the corpus from memory to free resources.
					Factory.deleteResource(corpus);
				}
				catch (Exception e)
				{
					logger.error("Error while processing corpus with name \"" 
							+ corpusNames[i] + "\" and Id=" + corpusIds[i] + ".", e);
					
					if(i<corpusNames.length-1)
					{
						try
						{
							// Reset the application
							application.interrupt();
							application.cleanup();
							application = (CorpusController)PersistenceManager.loadObjectFromFile(applicationFile);
						} 
						catch (Exception e2)
						{
							logger.error("Error resetting application after error.", e2);
						}
					}
				}
				logger.info("Finished processing corpus " + corpusNames[i] + ". Processing took " + (((double) (System.currentTimeMillis() - startMillis)) / 1000) + " seconds.");
				
			}
			logger.info("Batch processing of corpora finished. Processing took " + (((double) ((new Date()).getTime() - appStartMillis)) / 1000) + " seconds.");
		} 
		catch(Exception e)
		{
			logger.error("Unexpected error: " , e);
			logger.info("EXITING AFTER ERROR.");
			System.exit(1);
		}
		logger.info("Application finished normally.");
		if(documentsSkipped.size() > 0)
		{
			String msg = "";
			for (Document iDoc : documentsSkipped)
			{
				msg += iDoc.getName() + " "; 
			}
			msg = msg.trim();
			logger.warn("The following documents were not processed: " + msg);
		}
	}
	
	private static void parseCommandLineArgs(String[] args)
	{
		if(args.length <=0 || args[0] == null || args[0].isEmpty())
		{
			System.out.println("Usage: java GateWrapper CorpusName1 CorpusName2 ... CorpusNameN");
			System.out.println("\tAt least one corpus name should be specified");
			logger.error("Invalid arguments to shell command.");
			System.exit(1);
		}
		
		corpusNames = new String[args.length];
		corpusIds = new long[corpusNames.length];
		for(int i=0; i<args.length; i++)
		{
			corpusNames[i] = args[i];
		}
	}

	private static void readPropertiesFile()
	{
		Properties properties = new Properties();
	    try 
	    {
	        properties.load(new FileInputStream(PROPERTIES_FILE_NAME));
	    } 
	    catch (Exception e) 
	    {
	    	logger.warn("Error reading properties file " + PROPERTIES_FILE_NAME + ". Using defaults.", e);
	    }
	    datastoreUrl = properties.getProperty(DATASTORE_URL_PROPERTY_NAME, "jdbc:postgresql://localhost:5432/gatedb?user=gateuser&password=hd300d");

	    gateUser = properties.getProperty(GATE_USER_PROPERTY_NAME, "ADMIN");
	    gateGroup = properties.getProperty(GATE_GROUP_PROPERTY_NAME, "ADMINS");
	    gatePassword = properties.getProperty(GATE_PASSWORD_PROPERTY_NAME, "sesame");
		applicationFileName = properties.getProperty(GATE_APPLICATION_FILE_NAME_PROPERTY_NAME, ".\\ManhattanPipeline");
		gateConfigFileName =  properties.getProperty(GATE_CONFIGURATION_FILE_NAME_PROPERTY_NAME, ".\\gate.xml");
		logFile = properties.getProperty(LOG_FILE_NAME_PROPERTY_NAME);
		processingResourcesPath = properties.getProperty(PROCESSING_RESOURCES_PATH_PROPERTY_NAME);
		userPluginsPath = properties.getProperty(USER_PLUGINS_PATH_PROPERTY_NAME);
	}
	
	private static void init() throws IOException
	{
		logger = Logger.getLogger("GateWrapper");
		
		// Get the machine name we are running on
		String machineName = java.net.Inet4Address.getLocalHost().toString();
		Pattern machineNamePatt = Pattern.compile("^([A-Za-z0-9]+)");
		Matcher machineMatcher = machineNamePatt.matcher(machineName);
		if(machineMatcher.find())
			machineName = machineMatcher.group(1);
		
		// Get the database name.
		String dbName = "";
		Pattern dbNamePatt = Pattern.compile("\\/([^\\?\\/]+)\\?");
		Matcher dbMatcher = dbNamePatt.matcher(datastoreUrl);
		if(dbMatcher.find())
			dbName = dbMatcher.group(1);
		
		logFile = logFile.replace(".log", "-" + machineName + "-" + dbName + ".log");
		if(logger.getAppender("file") == null)
		{
			File file = new File(logFile);
			try
			{
				file.delete();
			} catch (Exception e) {}
			RollingFileAppender app = new RollingFileAppender();
			app.setFile(file.getAbsolutePath(), true, false, 1000);
			app.setMaxBackupIndex(5);
			app.setMaxFileSize("1000KB");
			PatternLayout layout = new PatternLayout();
			layout.setConversionPattern("%d{ABSOLUTE} %5p %c{1}:%L - %m%n");
			app.setLayout(layout);
			logger.addAppender(app);
		}
	}
	
	private static void getDatastoreSession() throws Exception
	{
		//open datastore
		dataStore = Factory.openDataStore(datastoreType, datastoreUrl);
		
		try
		{
			dataStore.open();
		}
		catch(gate.persist.PersistenceException pe)
		{
			logger.fatal("Cannot open database connection using Url=" + datastoreUrl, pe);
			System.exit(1);
		}

		//get security factory
		//the security factory should be initialised with the same JDBC url as the datastore
		//that's where the user/group information resides
		AccessController ac = Factory.createAccessController(datastoreUrl);
		ac.open();

		//login and get session
		User usr = ac.findUser(gateUser);
		Group grp = ac.findGroup(gateGroup);
		Session usrSession = ac.login(usr.getName(), gatePassword, grp.getID());
		if(! ac.isValidSession(usrSession))
		{
			throw new Exception("Cannot create session. Check gate credentials provided (gate.user, gate.group and gate.password).");
		}

		//use this session for all consequent operations with the datastore
		dataStore.setSession(usrSession);

		List ids = dataStore.getLrIds("gate.corpora.DatabaseCorpusImpl");
		for (Object id : ids)
		{
			String corpusName = dataStore.getLrName(id);
			for(int i=0; i<corpusNames.length; i++)
			{
				if(corpusNames[i].equalsIgnoreCase(corpusName) 
						|| corpusNames[i].equals(id.toString()))
				{
					corpusNames[i] = corpusName;
					corpusIds[i] = Long.parseLong(id.toString());
					break;
				}
			}
		}
		for(int i=0; i<corpusNames.length; i++)
		{
			if(corpusIds[i] <= 0)
			{
				throw new Exception("Corpus with name \"" + corpusNames[i] + "\" was not found in the database.");
			}
		}
	}
	
	private static void initGate() throws Exception
	{
//		try
//		{
//			File gateHomeDir = new File(System
//					.getProperty(GateConstants.GATE_HOME_PROPERTY_NAME));
//			File gateConfigFile = new File(gateHomeDir, "gate.xml");			
//			Gate.setGateHome(gateHomeDir);
//			Gate.setUserConfigFile(gateConfigFile);
//		}
//		catch(Exception e)
//		{
//			logger.warn("Error finding gate.xml. Is GATE_HOME environment variable set?", e);
//		}
		if(! (new File(userPluginsPath)).exists())
		{
			logger.fatal("Cannot find path " + userPluginsPath);
			System.exit(1);
		}
		if(! System.getProperties().containsKey("gate.home")
				|| ((String) System.getProperty("gate.home")).isEmpty() )
		{
			logger.warn("System property gate.home is not set or is empty."
					+ " Checking environment variable GATE_HOME.");
			gateHome = null;
			try
			{
				gateHome = System.getenv("GATE_HOME").trim();
			} 
			catch (Exception e){}
			if(gateHome == null || gateHome.isEmpty())
				logger.warn("Environment variable GATE_HOME is missing or empty.");				
			else
			{
				System.setProperty("gate.home", gateHome);
				logger.info("Environment variable GATE_HOME was found. Setting gate.home to " + gateHome);
			}
		}
		File userSessionFile= new File(".", "gate.session");		
		if(! userSessionFile.exists())
		{
			logger.fatal("Session file " + userSessionFile.getName() + " should be present in the current directory (" + System.getProperty("user.dir") + ").");
			System.exit(1);
		}
		else
		{
			setPathsInFile(userSessionFile);
		}
		File userConfigurationFile= new File(gateConfigFileName);
		if(! userConfigurationFile.exists())
		{
			logger.fatal("Configuration file " + userConfigurationFile.getName() + " should be present in the current directory (" + System.getProperty("user.dir") + ").");
			System.exit(1);
		}
		else
		{
			setPathsInFile(userConfigurationFile);
		}
		Gate.setUserSessionFile(userSessionFile);
		Gate.setUserConfigFile(userConfigurationFile);
		Gate.init();
	}
	
	private static Corpus loadCorpus(long corpusId) throws Exception
	{
		Corpus corpus;
		FeatureMap params = Factory.newFeatureMap();
		params.put(DataStore.DATASTORE_FEATURE_NAME, dataStore);
		params.put(DataStore.LR_ID_FEATURE_NAME, new Long(corpusId));
		corpus = (Corpus) Factory.createResource(gate.corpora.DatabaseCorpusImpl.class.getName(), params);
		return corpus;
	}
	
	private static void setPathsInFile(File file) throws Exception
	{
		FileReader reader = new FileReader(file);
		// Allocate 100Kb. Files should not be larger. 
		CharBuffer buffer = CharBuffer.allocate(100000);
		reader.read(buffer);
		reader.close();
		buffer.rewind();
		
		String fileContents = buffer.toString().trim();
		
		try
		{
			File f = new File(processingResourcesPath);
			if(f.exists()) processingResourcesPath = f.getAbsolutePath();
		} catch (Throwable t) {}

		try
		{
			File f = new File(gateHome);
			if(f.exists()) gateHome = f.getAbsolutePath();
		} catch (Throwable t) {}

		try
		{
			File f = new File(userPluginsPath);
			if(f.exists()) userPluginsPath = f.getAbsolutePath();
		} catch (Throwable t) {}

		String processingResourcesURL = processingResourcesPath.replace('\\', '/').replace(" ", "%20");
		String gateHomeURL = gateHome.replace('\\', '/').replace(" ", "%20");
		String userPluginsURL = userPluginsPath.replace('\\', '/').replace(" ", "%20");
		
		fileContents = fileContents.replaceAll("[a-zA-Z0-9-\\.\\/\\$\\{\\}\\%_\\:\\\\]*ProcessingResources", "file:///" + processingResourcesURL);
		fileContents = fileContents.replaceAll("[a-zA-Z0-9-\\.\\/\\$\\{\\}\\%_\\:\\\\]*GATE\\-4\\.0", "file:///" + gateHomeURL);
		fileContents = fileContents.replaceAll("[a-zA-Z0-9-\\.\\/\\$\\{\\}\\%_\\:\\\\]*GatePlugins", "file:///" + userPluginsURL);
		
		FileWriter writer = new FileWriter(file);
		writer.write(fileContents);
		writer.close();
	}
	
	private static void removeAnnotationsFromDoc(Document doc, String annotationNameList)
	{
		try
		{
			SimpleAnnotationSet annotSet = doc.getAnnotations();
			Iterator<Annotation> iter = annotSet.iterator();
			ArrayList<Annotation> annotsToRemove = new ArrayList<Annotation>();
			while(iter.hasNext())
			{
				Annotation annot = iter.next();
				if(annotationNameList.contains(annot.getType()))
					annotsToRemove.add(annot);
			}
			for (Annotation annotation : annotsToRemove)
			{
				annotSet.remove(annotation);
			}
		} 
		catch (Exception e)
		{
			logger.warn("Error while removing obsolete annotations from document.", e);
		}
	}
	
}
